<?php
/**
 * Created by PhpStorm.
 * User: dt
 * Time: 11:44
 */

namespace frontend\utils;

use Yii;
use yii\base\Exception;
use frontend\utils\CurlUtil;


/**
 * Url工具类
 * @package app\utils
 */
class UrlUtil
{
    //默认配置
    private static $defaults = [];
    //文件名称
    private static $truefile = null;
    //文件名称
    private static $fp_puts = null;
    //文件名称
    private static $fp_puts_false = null;
    //文件名称
    private static $fp_gets = null;
    //规则配置
    private static $coll_match = '';
    //类型配置
    private static $coll_type = '';
    //初始url
    private static $url = null;
    //错误url文件
    private static $errorFile = null;
    //正确url打开文件
    private static $true = null;
    //错误url打开文件
    private static $error = null;
    //url正则
    private static $match = "/^((http|https|ftp|telnet|news):\/\/)?([a-z0-9_\-\/\.]+\.[][a-z0-9:;&#@=_~%\?\/\.\,\+\-]+)$/";
    //配置条件
    private static $options = [];

    /**
     * 主程序
     */
    public static function start($url, $match, $replace)
    {
        self::$url = $url;
        self::folder($url);
        self::$coll_match = $match;
        self::$coll_type = $replace;
        self::$fp_puts_false = fopen(self::$errorFile, "ab");//记录url列表文件
        self::$fp_puts = fopen(self::$truefile, "ab");//记录url列表文件
        self::$fp_gets = fopen(self::$truefile, "r");//保存url列表文件
//        do {
            $result_url_arr = self::crawler();
            //文件存储
//            self::write($result_url_arr);
            //表存储
            return self::resetContent($result_url_arr);
//        } while ($current_url = fgets(self::$fp_gets, 1024));//不断获得url
    }

    /*
     * 数据整理
     */
    public static function resetContent($content,$one=1,$two=2)
    {
        $result = [];
        $sub_count = 0;
        if ($content) {
            foreach ($content as $k => $write_url) {
                $sub_count = count($write_url);break;
            }
        }
        for ($i=0;$i < $sub_count;$i++){
            $result[$i]['url'] = $content[$one][$i];
            $result[$i]['title'] = $content[$two][$i];
        }
        return $result;
    }

    /*
     * 项目文件夹
     */
    public static function folder($url)
    {
        $file_name = explode('.', $url)[1];
        if(!is_dir($file_name)){
            mkdir($file_name);
        }
        $truefile = $file_name.'/'.$file_name . '.txt';
        if (!file_exists($truefile)) file_put_contents($truefile, '');
        $errorFile = $file_name.'/'.$file_name . '_error.txt';
        if (!file_exists($errorFile)) file_put_contents($errorFile, '');
        self::$truefile = $truefile;
        self::$errorFile = $errorFile;
    }

    /*
     * 文件写入
     */
    public static function write($result_url_arr)
    {
        if ($result_url_arr) {
            foreach ($result_url_arr as $write_url) {
                if (preg_match(self::$match, $write_url)) {  //验证url是否正确
                    fputs(self::$fp_puts, $write_url . "\r\n");
                } else {
                    fputs(self::$fp_puts_false, $write_url . "\r\n");
                }
            }
        }
    }

    /*
     * 获取文件最后一行
     */
    public static function getEndLine()
    {
        fseek(self::$fp_gets,-1,SEEK_END);
        $s= '';
        while(($c = fgetc(self::$fp_gets)) !== false)
        {
            if($c == "\n" && $s) break;
            $s = $c . $s;
            fseek(self::$fp_gets, -2, SEEK_CUR);
        }
        if($s != ''){
            self::$url = $s;
        }
        return $s;
    }

    /**
     * 爬虫
     */
    public static function crawler()
    {
        $content = self::getUrlContent();
        if ($content) {
            $result_content = self::filterContent($content);
            $url_list = self::reviseUrl(self::$url, $result_content[1]);
            if (!is_dir(self::$url)) {
                if (is_array($url_list)) {
                    foreach ($url_list as $key => $value) {
                        mkdir(self::$url . '/' . $key);
                    }
                }
            }
            $url_list = self::classification($url_list);
            $result_content[1] = $url_list;
            return $result_content;
        }
    }

    /*
     * 去掉无用的url
     */
    public static function classification($url_list)
    {
        if ($url_list) {
            foreach ($url_list as $k => $val){
                if (strpos($val, self::$coll_type) === false) {
                    unset($url_list[$k]);
                }
            }
            return $url_list;
        }
    }

    /**
     * 获取url的html
     */
    public static function getUrlContent()
    {
        try {
            $content = file_get_contents(self::$url);
            return $content;
        } catch (\Exception $e) {
            $fp_puts = fopen("error.txt", "ab");//记录url列表
            $fp_gets = fopen("error.txt", "r");//保存url列表
            fputs($fp_puts, self::$url . "\r\n");
        }
    }

    /**
     * 修正相对路径
     */
    public static function reviseUrl($base_url, $url_list)
    {
        $url_info = parse_url($base_url);
        $base_url = $url_info["scheme"] . '://';
        if (!empty($url_info["user"]) && !empty($url_info["pass"])) {
            if ($url_info["user"] && $url_info["pass"]) {
                $base_url .= $url_info["user"] . ":" . $url_info["pass"] . "@";
            }
        }
        $base_url .= $url_info["host"];
        if (!empty($url_info["port"])) {
            if ($url_info["port"]) {
                $base_url .= ":" . $url_info["port"];
            }
        }
        if (!empty($url_info["path"])) {
            $base_url .= $url_info["path"];
        }
        if (is_array($url_list)) {
            foreach ($url_list as $url_item) {
                if (preg_match('/^http/', $url_item)) {
                    //已经是完整的url
                    $result[] = $url_item;
                } else {
                    //不完整的url
                    $real_url = $base_url . '/' . $url_item;
                    $result[] = $real_url;
                }
            }
            return $result;
        } else {
            return;
        }
    }

    /**
     * 从html内容中筛选链接及有用信息
     */
    public static function filterContent($web_content)
    {
        $reg_tag_a = self::$coll_match;
        $result = preg_match_all($reg_tag_a, $web_content, $match_result);
        if ($result) {
            return $match_result;
        }
    }
}

?>